# -*- coding:utf-8 -*-
import re
import numpy as np
import lxml_html_extension

class CleanFeature(object):

    key_words_list = ["share", "button", "icon", "twitter", "facebook", "email", "tags", "odds", "tweet", "comment",
                      "linkedin", "comments", "thumbnail", "author", "google", "gallery", "google+", "photo",'pinterest',
                      'click','post','copyright','widget']

    def __init__(self,node):
        self.node = node

    def get_hstack(self):
        h = np.hstack([self.node.tag, self.node.rect_position,self.own_area()])
        h = np.hstack([h,self.anchor_text_feature()])
        h = np.hstack([h,self.img_feature()])
        h = np.hstack([h,self.a_tag_img_feature()])
        h = np.hstack([h,self.iframe_feature()])
        h = np.hstack([h,self.ul_tag_feature()])
        h = np.hstack([h,self.ins_tag_feature()])
        h = np.hstack([h,self.script_tag_feature()])
        h = np.hstack([h,self.nearby_p_tag_feature()])
        h = np.hstack([h,self.tag_name_feature()])
        h = np.hstack([h,self.own_visual_status()])
        h = np.hstack([h,self.form_tag_feature()])
        h = np.hstack([h,self.key_words_feature()])
        return h.astype('str')

    @classmethod
    def get_column_name(cls):
        column_name_list = list()
        column_name_list += ['tag_name','left','right','top','bottom','own_area']
        column_name_list += ['total_anchor_text_length','total_text_count','anchor_density','anchor_tag_count','without_anchor_text_length']
        column_name_list += ['img_count','biggest_img_area','img_total_area','biggest_img_left','biggest_img_right','biggest_img_top','biggest_img_bottom']
        column_name_list += ['a_tag_img_count','a_tag_biggest_img_area','a_tag_img_total_area','a_tag_biggest_img_left','a_tag_biggest_img_right','a_tag_biggest_img_top','a_tag_biggest_img_bottom']
        column_name_list += ['iframe_count','biggest_iframe_area','iframe_left','iframe_right','iframe_top','iframe_bottom']
        column_name_list += ['ul_count','biggest_ul_area','ul_left','ul_right','ul_top','ul_bottom','li_count', 'a_tag_li_count', 'min_li_text_count', 'max_li_text_count']
        column_name_list += ['ins_count','script_count']
        column_name_list += ['previous_p_distance','previous_p_text_count','next_p_distance','next_p_text_count']
        column_name_list += ['tag_name_count','uniq_tag_name_count']
        column_name_list += ['display','visibility']
        column_name_list += ['form_count','input_count','button_count']
        column_name_list += ['%s_text' % i for i in CleanFeature.key_words_list]
        column_name_list += ['%s_attrib' % i for i in CleanFeature.key_words_list]
        return column_name_list

    def own_visual_status(self):
        display = 0 if self.node.attrib.get('vips_display', 'inline') == 'none' else 1
        visibility = 0 if self.node.attrib.get('vips_visibility', 'visible') in ('hidden', 'collapse') else 1
        return display,visibility
#返回0/1，标签是否有预览块
    def own_area(self):
        t_l, t_r, t_t, t_b = self.node.rect_position
        area = (t_r - t_l) * (t_b - t_t)
        return area
#返回数字，标签面积
    def ins_tag_feature(self):
        ins_children = self.node.xpath('.//ins')
        return len(ins_children)
#返回整数，标签里存在ins广告标签数量
    def form_tag_feature(self):
        form_count = len(self.node.xpath('.//form'))
        input_count = len(self.node.xpath('.//input'))
        button_count = len(self.node.xpath('.//button'))
        return form_count,input_count,button_count
#返回三个整数，标签里存在表单数量
    def script_tag_feature(self):
        script_children = self.node.xpath('.//script')
        return len(script_children)
#返回整数，标签里存在script标签数量
    def iter_p_tag(self,preceding_status=False):
        seq = 0
        for n in self.node.itersiblings(preceding=preceding_status):
            seq += 1
            if n.tag == 'p' and len(n.text_content().strip()) > 0:
                p_text_count = len(n.text_content().strip())
                return seq,p_text_count
        return 0,0
#返回两个整数，同级p标签数量以及其长度
    def nearby_p_tag_feature(self):
        previous_p_distance,previous_p_text_count = self.iter_p_tag(True)
        next_p_distance,next_p_text_count = self.iter_p_tag(False)
        return previous_p_distance,previous_p_text_count,next_p_distance,next_p_text_count
#返回四个整数，该p标签前的p标签数以及长度，该p标签后的p标签数以及长度
    def tag_name_feature(self):
        tag_name_list = [self.node.tag]
        for i in self.node.iterdescendants():
            tag_name_list.append(i.tag)
        tag_name_count = len(tag_name_list)
        uniq_tag_name_count = len(set(tag_name_list))
        return tag_name_count,uniq_tag_name_count
#返回两个整数，该标签内子标签的数量与去重后标签数量。
    def anchor_text_feature(self):
        anchor_children = self.node.xpath('.//a')
        total_anchor_text_list = list()
        anchor_text_length = 0
        for a in anchor_children:
            anchor_text = a.text_content().strip()
            anchor_text_length += len(anchor_text)
            total_anchor_text_list.append(anchor_text)
        total_text_count = len(self.node.text_content().strip())
        anchor_density = anchor_text_length*1.0/total_text_count if total_text_count > 0 else 0
        total_anchor_text_length = len(''.join(total_anchor_text_list))
        anchor_tag_count = len(anchor_children)
        without_anchor_text_length = total_text_count - total_anchor_text_length
        return total_anchor_text_length,total_text_count,anchor_density,anchor_tag_count,without_anchor_text_length
#返回五个数，文本内a标签的长度，全部文本长度，文本长度数大于0返回文本长度和a标签文本长度的比例小于则为0，文本内a标签总数，总文本长度与文本内a标签内容长度的差。
    def a_tag_img_feature(self):
        img_children = self.node.xpath('.//a//img')
        img_total_area = 0
        img_count = len(img_children)
        biggest_img_area = 0
        biggest_img_left,biggest_img_right,biggest_img_top,biggest_img_bottom = 0,0,0,0
        for i in img_children:
            t_l, t_r, t_t, t_b = i.rect_position
            img_area = (t_b-t_t) * (t_r-t_l)
            img_total_area += img_area
            if img_area > biggest_img_area:
                biggest_img_area = img_area
                biggest_img_left, biggest_img_right, biggest_img_top, biggest_img_bottom = t_l, t_r, t_t, t_b
        return img_count,biggest_img_area,img_total_area,biggest_img_left, biggest_img_right, biggest_img_top, biggest_img_bottom
#返回七个整数，正文内的带连接的img数量，最大带连接的img的面积，带连接的img标签总面积，最大带连接的img的左.右.上.下距离

    def img_feature(self):
        img_children = self.node.xpath('.//img')
        img_total_area = 0
        img_count = len(img_children)
        biggest_img_area = 0
        biggest_img_left,biggest_img_right,biggest_img_top,biggest_img_bottom = 0,0,0,0
        for i in img_children:
            t_l, t_r, t_t, t_b = i.rect_position
            img_area = (t_b-t_t) * (t_r-t_l)
            img_total_area += img_area
            if img_area > biggest_img_area:
                biggest_img_area = img_area
                biggest_img_left, biggest_img_right, biggest_img_top, biggest_img_bottom = t_l, t_r, t_t, t_b
        return img_count,biggest_img_area,img_total_area,biggest_img_left, biggest_img_right, biggest_img_top, biggest_img_bottom
#返回七个整数，正文内的img数量，最大图片的面积，img标签总面积，最大图片的左.右.上.下距离

    def iframe_feature(self):
        iframe_children = self.node.xpath('.//iframe')
        iframe_count = len(iframe_children)
        i_l,i_r,i_t,i_b = 0,0,0,0
        biggest_iframe_area = 0
        for i in iframe_children:
            t_l, t_r, t_t, t_b = i.rect_position
            iframe_area = (t_b - t_t) * (t_r - t_l)
            if iframe_area > biggest_iframe_area:
                biggest_iframe_area = iframe_area
                i_l, i_r, i_t, i_b = t_l, t_r, t_t, t_b
        return iframe_count,biggest_iframe_area,i_l,i_r,i_t,i_b
#返回六个整数，标签中iframe数量，最大的iframe标签面积，最大iframe标签的左.右.上.下距离

    def ul_tag_feature(self):
        ul_children = self.node.xpath('.//ul')
        ul_count = len(ul_children)
        i_l,i_r,i_t,i_b = 0,0,0,0
        biggest_ul_area = 0
        biggest_ul = None
        for i in ul_children:
            t_l, t_r, t_t, t_b = i.rect_position
            iframe_area = (t_b - t_t) * (t_r - t_l)
            if iframe_area > biggest_ul_area:
                biggest_ul = i
                biggest_ul_area = iframe_area
                i_l, i_r, i_t, i_b = t_l, t_r, t_t, t_b
        li_count, a_tag_li_count, min_li_text_count, max_li_text_count = self.li_tag_feature(biggest_ul) if biggest_ul is not None else (0,0,0,0)
        return ul_count,biggest_ul_area,i_l,i_r,i_t,i_b,li_count, a_tag_li_count, min_li_text_count, max_li_text_count
#返回十个整数，标签中ul数量，占面积最大的ul面积，最大ul标签的左.右.上.下距离，最大ul下的li数量，最大ul下li下的a标签数量，最大ul的li子标签内最少文本数量，最大ul的li子标签内最大文本数量。如果没有ul则返回全是0.
    def li_tag_feature(self,ul_node):
        li_children = ul_node.xpath('.//li')
        a_tag_li_children = ul_node.xpath('.//li//a')
        li_count = len(li_children)
        a_tag_li_count = len(a_tag_li_children)
        min_li_text_count = 999
        max_li_text_count = 0
        for i in li_children:
            li_text_length = len(i.text_content())
            if li_text_length > max_li_text_count:
                max_li_text_count = li_text_length
            elif li_text_length < min_li_text_count:
                min_li_text_count = li_text_length
        return li_count, a_tag_li_count,min_li_text_count,max_li_text_count
#返回四个整数，该ul下的li数量，该ul下li下的a标签数量，li子标签内最少文本数量，li子标签内最大文本数量。
    def calculate_attrib_key_words_count(self, node, key_words_dict):
        for attrib_name in node.attrib:
            atttib_value = re.sub(r'([A-Z]+)', r'_\1', node.attrib.get(attrib_name,''))
            #正则替换
            for k in CleanFeature.key_words_list:
                match_items = re.findall(k,atttib_value)
                #计算atttib_value里有多少个k，并传回一个数组。
                key_words_dict[k] = key_words_dict.get(k,0) + len(match_items)
        return key_words_dict
#返回一个字典，字典的每一个key的值为整数       代表与这个key的属性出现过的次数。
    def calculate_text_key_words_count(self, node, key_words_dict):
        text_content = node.text_content()
        atttib_value = re.sub(r'([A-Z]+)', r'_\1', text_content)
        for k in CleanFeature.key_words_list:
            if k in atttib_value:
                key_words_dict[k] = key_words_dict.get(k,0) + 1
        return key_words_dict
#返回一个字典，字典的每一个key的值位整数       代表和key相同的文本+1
    def key_words_feature(self):
        attrib_key_words_dict = dict()
        text_key_words_dict = dict()
        self.calculate_text_key_words_count(self.node, text_key_words_dict)
        #返回字典记录相同的文本
        self.calculate_attrib_key_words_count(self.node, attrib_key_words_dict)
        #返回字典记录相同的属性出现次数
        for n in self.node.iterdescendants():
            self.calculate_attrib_key_words_count(n, attrib_key_words_dict)
            #便利后代中的属性是否和数组中不同
        text_key_words_feature = list()
        attrib_key_words_feature = list()
        for i in CleanFeature.key_words_list:
            text_key_words_feature.append(text_key_words_dict.get(i,0))
            attrib_key_words_feature.append(attrib_key_words_dict.get(i,0))
            #把每一个CleanFeature.key_words_list中的项便利到两个新的list中
        key_words_feature = text_key_words_feature + attrib_key_words_feature
        return key_words_feature
#返回一个list，list为在CleanFeature.key_words_list中已有的相同的文本和不同的属性。

if __name__ == '__main__':
    with open('test_clean_feature','w') as tf:
        tf.write('url\txpath\t%s\n' % '\t'.join(CleanFeature.get_column_name()))
        url = 'a_test'
        html_content = open('test.html').read()
        content_xpath = '//div[@id="content-area"]'
        root_node = lxml_html_extension.fromstring(html_content)
        content_dom_list = root_node.xpath(content_xpath)
        if len(content_dom_list) == 1:
            content_dom = content_dom_list[0]
            for p_p in content_dom.getchildren():
                if not isinstance(p_p, lxml_html_extension.HtmlElement):
                    continue
                xpath = p_p.custom_xpath()
                article_feature_obj = CleanFeature(p_p)
                feature_hstack = '\t'.join(article_feature_obj.get_hstack().tolist())
                tf.write('%s\t%s\t%s\n' % (url, xpath, feature_hstack))